# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request, FormRequest
import re
import urllib.request
import urllib.parse
from zhihu.items import ZhihuItem

class ZhSpider(scrapy.Spider):

	uapools = [
    'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0'
    ]

	def ua(self, uapools):
	    thisua = random.choice(uapools)
	    print(thisua)
	    headers = ('User-Agent', thisua)
	    opener = urllib.request.build_opener()
	    opener.addheaders = [headers]

	    #安装为全局
	    urllib.request.install_opener(opener)

	name = 'zh'
	header = {
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
	}

	header1 = {
		'X-Requested-With': 'XMLHttpRequest',
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
		'Referer': 'https://www.zhihu.com/topics',
		'X-Xsrftoken':'2f5ec99e486b00ce7ac62f4bc3a66d36',
		'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
		'Cookie': 'd_c0="ADAAITZRQAqPTrRJ0UCQETn1rADBW-w6ECg=|1468894948"; _za=25c668d4-929c-415c-ac5c-b3b65c3a92b7; _zap=1f7056ae-9102-46a9-b86c-3a85d38881e3; q_c1=745606ff5de54ea497a0d0beca17d3f5|1507861778000|1468894948000; q_c1=745606ff5de54ea497a0d0beca17d3f5|1516153079000|1468894948000; _xsrf=2f5ec99e486b00ce7ac62f4bc3a66d36; capsion_ticket="2|1:0|10:1517280905|14:capsion_ticket|44:ZWU0YjJlMzVlYjZhNGMxNjk0MTRlMjc1Mjg0YWE0Njk=|bcf2f422fff33c8ba655764af8775bf80f4570fd0277bc859befb1f011171653"; aliyungf_tc=AQAAAP2oDi4LSwwAar3idO3QFFCxxVIu; l_cap_id="NjgzMTIyYmZjMzBjNDAxOTk5YWVkZDNiZjg4ZDk1OTU=|1517368208|3fa55ca92e01006661146c8389cc59893dd5e757"; r_cap_id="ZGQ5MzU4Yzg3MzBjNDRiNmFiNGRkMGI4OWE3ZTliZGI=|1517368208|3069b469b605c319851b7d321f100aff98ac8323"; cap_id="OTMxNGEyZDQwNDEyNGVjYWFiMTdjYWFmZDQ2MDc3ZWU=|1517368208|9a2e2f846209a2e2588b0eabf29d7c50fef0fad0"; __utma=51854390.1822274323.1517280939.1517280939.1517368209.3; __utmb=51854390.0.10.1517368209; __utmc=51854390; __utmz=51854390.1517280939.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmv=51854390.000--|3=entry_date=20160719=1'
	}
	header2 = {
		'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36',
		'Cookie': 'd_c0="ADAAITZRQAqPTrRJ0UCQETn1rADBW-w6ECg=|1468894948"; _za=25c668d4-929c-415c-ac5c-b3b65c3a92b7; _zap=1f7056ae-9102-46a9-b86c-3a85d38881e3; q_c1=745606ff5de54ea497a0d0beca17d3f5|1507861778000|1468894948000; capsion_ticket="2|1:0|10:1517819719|14:capsion_ticket|44:NzM1YTE1MzAwZDlmNDJjZWJmMGMwZjc4YjViMWMyNGE=|365f8de283550a5a66451ec3d678e885379d6ecd585a3e8aeabf6a4a7f705e2d"; q_c1=745606ff5de54ea497a0d0beca17d3f5|1519614962000|1468894948000; aliyungf_tc=AQAAAD4Qol0v8gcAar3idFaa4K6lTrQu; _xsrf=7ebef6f483a6f6ded336766be3986ae1; l_cap_id="NzJlMmExNTVkYTU5NDNlZWIxODAxYzliMjQ3NDkyZDQ=|1520303579|b94aaa441672a83e234a0c66d7b33b4f6baf6c89"; r_cap_id="NDkzYzBlZDhmNjQxNDA5MzhlNjg0ZGI2ODBiNzZmMzQ=|1520303579|f6f64e797fb8085839a2e6ed22ec33afe22c2f82"; cap_id="MGRkZDkxOTBhYzExNGI1ZDhjZjg4NDg1OGM0MmNlMjc=|1520303579|fb3e79b41f1e4995f0d9737e8390adcd0fe0e50a"; __utma=51854390.14302589.1519872536.1519872536.1520303582.2; __utmb=51854390.0.10.1520303582; __utmc=51854390; __utmz=51854390.1519872536.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmv=51854390.000--|3=entry_date=20160719=1',
		'Referer': 'https://www.zhihu.com/topic/19550994',
	}
	allowed_domains = ['zhihu.com']
	classifyAll = ['游戏', '运动', '互联网', '艺术', '阅读', '美食', '动漫', '汽车', '生活方式', '教育', '摄影', '历史', '文化', '旅行', '职业发展', '经济学', '足球', '篮球', '投资', '音乐', '电影', '法律', '自然科学', '设计', '创业', '健康', '商业', '体育', '科技', '化学', '物理学', '生物学', '金融']
	#start_urls = ['http://zhihu.com/']
	def start_requests(self):
		# url = 'https://www.zhihu.com/api/v4/topics/19550994/feeds/top_activity?include=data%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Danswer%29%5D.target.content%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Danswer%29%5D.target.is_normal%2Ccomment_count%2Cvoteup_count%2Ccontent%2Crelevant_info%2Cexcerpt.author.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Darticle%29%5D.target.content%2Cvoteup_count%2Ccomment_count%2Cvoting%2Cauthor.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dtopic_sticky_module%29%5D.target.data%5B%3F%28target.type%3Dpeople%29%5D.target.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.content%2Crelationship.is_authorized%2Cis_author%2Cvoting%2Cis_thanked%2Cis_nothelp%3Bdata%5B%3F%28target.type%3Danswer%29%5D.target.author.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Darticle%29%5D.target.content%2Cauthor.badge%5B%3F%28type%3Dbest_answerer%29%5D.topics%3Bdata%5B%3F%28target.type%3Dquestion%29%5D.target.comment_count&limit=5&after_id=4438.94283'
		# Request(url, headers = self.header, encoding='utf-8', cookies={'zhihu.com': 'true'}, callback = self.aaa)
		# return [Request('',
		# 	headers = self.header2,encoding='utf-8', callback = self.aaa)]
		for classify in self.classifyAll:
			yield Request('https://www.zhihu.com/topics#'+classify, headers = self.header1, encoding='utf-8', cookies={'zhihu.com': 'true'}, callback = self.getClassify)
		return[]

	'''获取话题分类'''
	def getClassify(self, response):
		className = response.xpath('//li[@class="zm-topic-cat-item"]/a/@href').extract()
		classId = response.xpath('//li[@class="zm-topic-cat-item"]/@data-id').extract()
		print(classId)

		for cid in classId:
			print('正在爬取'+cid+'的数据，请稍侯。。。')
			offset = 0
			while(True):

				formData = {
					"method": "next",
					"params": '{"topic_id": '+ cid +',"offset": '+ str(offset) +',"hash_id": ""}'
				}
				
				#post请求数据
				formData = bytes(urllib.parse.urlencode(formData), encoding='utf-8')

				data = urllib.request.urlopen('https://www.zhihu.com/node/TopicsPlazzaListV2', data = formData).read().decode('utf-8')
				body = eval(data)
				content = body['msg']
				if len(content) == 0:
					break
				pat = '<a target="_blank" href="(.*?)"'
				pat2 = '<img src=".*?" alt="(.*?)"'

				for i in content:
					link = re.compile(pat, re.S).findall(i)
					url = 'https://www.zhihu.com' + re.sub('\\\/', '/', link[0]) + '/hot'
					name = re.compile(pat2, re.S).findall(i)
					item = ZhihuItem()
					item['className'] = name[0]
					item['classUrl'] = url
					yield item
					# yield Request(url, headers = self.header, encoding='utf-8', cookies={'zhihu.com': 'true'}, callback = self.parse)
				offset += 20
		# https://www.zhihu.com/topic/19552706/hot
		# yield FormRequest('http://www.zhihu.com/node/TopicsPlazzaListV2',
		# 	headers = self.header1,
		# 	encoding = 'utf-8',
		# 	formdata = data,
		# 	callback = self.parse, 
		# 	)
		return []

	def parse(self, response):
		print(response.url)
		# print(response.body)
		body = response.body.decode('utf-8')
		# content = body['msg']
		
		pat = 'after_id=(.*?)&quot;'
		after_id = re.compile(pat, re.S).findall(body)
		print(after_id)
		# print(link)
		# print(content)
